import pandas as pd
from pandas import Series, DataFrame
import numpy as np
import matplotlib.pyplot as plt 
import chardet
import folium as fm
from folium import Marker, GeoJson
from folium.plugins import MarkerCluster, HeatMap, StripePattern
import geopandas as gpd
from geopandas import GeoSeries
from shapely.geometry import Point, LineString
import branca as br
import pandas as pd
from pandas import Series, DataFrame
import numpy as np
import matplotlib.pyplot as plt 
import chardet
import pandas as pd
import numpy as np
import folium
from folium.plugins import BeautifyIcon
import branca
import folium
from folium.plugins import MarkerCluster
from folium import IFrame
import shapely
from shapely.geometry import Point

import pandas as pd
import geopandas as gpd
from geopandas.tools import sjoin

import base64
import math
import os
import numpy as np
import seaborn as sns
import unicodedata
import pysal as ps
import branca
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver import ActionChains
import pandas as pd
import numpy as np
import os
import time
import re
import unidecode
import time 
from selenium.webdriver.support.ui import Select
from webdriver_manager.chrome import ChromeDriverManager
import re
---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
~\AppData\Local\Temp\ipykernel_10112\208844336.py in <module>
      4 import matplotlib.pyplot as plt
      5 import chardet
----> 6 import folium as fm
      7 from folium import Marker, GeoJson
      8 from folium.plugins import MarkerCluster, HeatMap, StripePattern

ModuleNotFoundError: No module named 'folium'

1. Intersecting Institutes points with Shapefiles#

inst = pd.read_csv( r'../_data/institutos1.csv' )

inst_geo = gpd.GeoDataFrame( inst, crs = "EPSG:4326", 
                                 geometry = gpd.points_from_xy( inst.nlong_ie, 
                                                             inst.nlat_ie ) )

shp_dpt = gpd.read_file( r"..\_data\INEI_LIMITE_DEPARTAMENTAL\INEI_LIMITE_DEPARTAMENTAL.shp" )

intersct_inst_geo = gpd.overlay( inst_geo , shp_dpt , how = 'intersection' )

# variables: meta de admision, total postulantes, total ingresantes, total matriculados
vars_sum = [ 'ltimoden_metaatencion', 'cuentadeid_postulante_procesoadm',
       'sumaden_flagingresante', 'sumaden_flagmatriculado' ]

tot_dpt = intersct_inst_geo.groupby( ['CCDD'], as_index = False )[ vars_sum ].sum()

tot_dpt_shp = shp_dpt.merge( tot_dpt, on = 'CCDD' )
import json
import requests
import spectra

from matplotlib.colors import rgb2hex

Generate colors

n_categories = 4

full_palette = sns.color_palette('PiYG', n_colors=(n_categories - 1) * 2 + 1)
cmap_x = full_palette[n_categories - 1:]
sns.palplot(cmap_x)

cmap_y = list(reversed(full_palette))[n_categories - 1:]
sns.palplot(cmap_y)
_images/51e831e60b2b1809643f7514acc4aad863d25860c18b3a996c39fda689d0d32f.png _images/a7a810ccc74da55a4d1af9c9fa952df46da5c1af215a7135ae03aba61c694a29.png

Generate matrix of colors

cmap_xy = []
bivariate_palette = {}

for j in range(n_categories):
    for i in range(n_categories):
        x = spectra.rgb(*cmap_x[i][0:3])
        y = spectra.rgb(*cmap_y[j][0:3])
        
        if i == j and i == 0:
            cmap_xy.append(x.darken(1.5).rgb)
        elif i == 0:
            cmap_xy.append(y.rgb)
        elif j == 0:
            cmap_xy.append(x.rgb)
        else: 
            blended = x.blend(y, ratio=0.5)
            
            if i == j:
                blended = blended.saturate(7.5 * (i + 1))
            else:
                blended = blended.saturate(4.5 * (i + 1))
                
            cmap_xy.append(blended.rgb)
            
        bivariate_palette[(i, j)] = rgb2hex(cmap_xy[-1])
        print(j, i)
            
cmap_xy = np.array(cmap_xy).reshape(n_categories, n_categories, 3)
0 0
0 1
0 2
0 3
1 0
1 1
1 2
1 3
2 0
2 1
2 2
2 3
3 0
3 1
3 2
3 3
plt.imshow(cmap_xy)
<matplotlib.image.AxesImage at 0x1d5034308b0>
_images/9e2cbb49840fc9630ddbe357a0e96f44d5dee09bd82b1e71b61d881bab8654f2.png
tot_dpt.head()
CCDD ltimoden_metaatencion cuentadeid_postulante_procesoadm sumaden_flagingresante sumaden_flagmatriculado ltimoden_metaatencion_qt cuentadeid_postulante_procesoadm_qt sumaden_flagingresante_qt sumaden_flagmatriculado_qt color
0 01 1213 1346 925 917 1 1 1 1 #efe3b6
1 02 2969 4007 2535 2501 2 2 2 2 #c6b268
2 03 1647 2997 1506 1506 1 2 1 1 #efbeb8
3 04 3000 2829 1976 1941 2 2 2 2 #c6b268
4 05 3107 3824 2406 2374 2 2 2 2 #c6b268
# Generate quartiles for data
for vr in vars_sum:
    tot_dpt[ f'{vr}_qt'] = pd.qcut(tot_dpt[ vr ], 4, [0, 1, 2, 3])

# Assign colors
tot_dpt['color'] = tot_dpt.apply( lambda x: bivariate_palette[tuple(x[['ltimoden_metaatencion_qt', 'cuentadeid_postulante_procesoadm_qt']].tolist())], axis = 1 )
# merge with data
merge1 = shp_dpt.merge(tot_dpt, on = 'CCDD' )

# Generate Json
shp_dpt_json = merge1.to_json(  )
shp_dpt_json[ : 10000]
# Generate function to assign color
style_function = lambda x: {
    'fillColor' : x['properties']['color'],
    "fillOpacity": 1,
        "weight": 0,
}
m1 = folium.Map(location = [ -12.0757538, -76.9863174  ], tiles="cartodbpositron", zoom_start=3)
folium.GeoJson(shp_dpt_json, style_function = style_function).add_to(m1)

m1
Make this Notebook Trusted to load map: File -> Trust Notebook

Choropleth Map

# government palace coordinates

lat_palacio = -12.0757538
long_palacio = -76.9863174

z = fm.Map(location = [lat_palacio, long_palacio], tiles='cartodbpositron', zoom_start = 5)

# Mandatory: geo_data in GeoJson format
# columns: variables from economics indicators data set
# Atention !!! key_on: commom variable between geodata and data "feature.properties.(name of variable)"


fm.Choropleth(
    geo_data=shp_dpt,
    data = tot_dpt_shp,
    columns=['OBJECTID', 'ltimoden_metaatencion'],
    key_on="feature.properties.OBJECTID",
    fill_color="YlOrRd",
    fill_opacity=0.8,
    line_opacity=0.2,
    legend_name="Total Vacancies",
    smooth_factor=0,
    Highlight= True,
    line_color = "#0000",
    overlay=True,
    nan_fill_color = "White"  # fill white missing values 
    ).add_to(z)

fm.LayerControl().add_to(z)

# Save in a html format 

# z.save("Poverty_Map.html")

z
Make this Notebook Trusted to load map: File -> Trust Notebook

Interactive Map

# Function create table by each Health center using html. This funtion will be aplly by each row
# Almost alway each code on html requires a beginnig <p> and ending </p> 

def visual_html( data, iterator, right_color, left_color, dict_cols ):
 
    # information by Health center
    values = []
    for col in dict_cols['old_names']:
        values.append( data[ col ].iloc[ iterator ] )
        
    # Generate table
    df_table1 = pd.DataFrame( { "Variables" : dict_cols['new_names'] , "Values" : values } )
    
    # Generate dict of colors
    color_col= dict(zip( df_table1.columns, ['background-color:'+i for i in [right_color, left_color] ]))
    
    # Define colors data
    def mycolor( data ):
        style = pd.DataFrame( color_col, index = data.index, columns = data.columns )
        df1 = data.mask( data.replace( '', np.nan ).notna(), style )
        return df1
    
    # Apply color to table
    df_final = df_table1.style.apply(mycolor, axis = None ) \
                        .hide( axis = 'index') \
                        .hide( axis = 'columns') \
                        .set_properties(**{
                                            'font-size': '15pt',
                                            })
    
    html = df_final.to_html()

    return html
col_dict = { 'old_names' : vars_sum, 'new_names' : [ 'Total Vacancies', 
                                         'Total Applicants', 
                                         'Total Entrants', 
                                         'Total Enrolled' ] }
ubication = inst['nlat_ie'].mean(), inst['nlong_ie'].mean()  # Average point

sol = fm.Map( location = ubication, zoom_start=12)

for i in range(0,len(inst)):
    html = visual_html( inst, 
                       iterator = i , 
                       right_color= '#F7E7A6 ', 
                       left_color ='#31BFD6', 
                       dict_cols = col_dict )

    iframe = br.element.IFrame(html=html,width=220,height=150)
    popup = fm.Popup(iframe,parse_html=True)
    
    fm.Marker( [inst['nlat_ie'].iloc[i],inst['nlong_ie'].iloc[i]],
                  popup = popup, 
                  icon = fm.Icon( color = 'blue', icon = 'university', prefix="fa")
             ).add_to(sol)

sol
Make this Notebook Trusted to load map: File -> Trust Notebook

Cluster

inst[ 'ratio_applicants_vacancies' ] = inst.cuentadeid_postulante_procesoadm / inst.ltimoden_metaatencion
inst.loc[ inst.ratio_applicants_vacancies > 1, 'Applicants_Vacancies' ] = "Applicant Surplus"
inst.loc[ inst.ratio_applicants_vacancies <= 1, 'Applicants_Vacancies' ] = "Applicant Deficit"

inst_sup = inst[inst['Applicants_Vacancies'] == "Applicant Surplus"].copy()
inst_def = inst[inst['Applicants_Vacancies'] == "Applicant Deficit"].copy()
location_sup = list( zip( inst_sup['nlat_ie'], inst_sup ['nlong_ie'] ) )
location_def = list( zip( inst_def['nlat_ie'], inst_def ['nlong_ie'] ) )
# Get lat and long health_center
zoom_start = 8
m = folium.Map( location= ubication, zoom_start=zoom_start)

### Applicant Surplus
marker_cluster_1 = MarkerCluster(
    locations=location_sup,
    name = "Applicant Surplus",
    overlay=True,
    control=True,
    
)

# Applicant Deficit
marker_cluster_2 = MarkerCluster(
    locations=location_def,
    name="Applicant Deficit",
    overlay=True,
    control=True,
    
)




marker_cluster_1.add_to(m)
marker_cluster_2.add_to(m)


# folium.TileLayer('openstreetmap').add_to(m)
# folium.TileLayer('Stamen Terrain').add_to(m)

folium.LayerControl().add_to(m)
m

Circle Markers

inst.loc[ inst.ratio_applicants_vacancies > 1, 'Applicants_Vacancies' ] = "Applicant Surplus"
inst.loc[ inst.ratio_applicants_vacancies <= 1, 'Applicants_Vacancies' ] = "Applicant Deficit"

inst_sup = inst[inst['Applicants_Vacancies'] == "Applicant Surplus"].copy()
inst_def = inst[inst['Applicants_Vacancies'] == "Applicant Deficit"].copy()
inst_def
# Import shapefile
geo_distrito = gpd.read_file( r'../_data/shape_file/DISTRITOS.shp')
# Get lat and long health_center
zoom_start = 8

m = folium.Map( location = ubication, zoom_start=zoom_start)

# # Add District shapefiles
# style2 = {'fillColor': '#ffffff', 'lineColor': '#00FFFFFF'}
# folium.GeoJson( data = geo_distrito[ "geometry" ], 
#                 style_function = lambda x:style2, 
#                 name = 'District Border'
#               ).add_to( m )


# Genereate groups in folioum
health_centers_group = folium.FeatureGroup( name = 'Heakth Centers', control=True)

inst_sup_fm = folium.FeatureGroup( name = 'Applicant Surplus', control = True )
inst_def_fm = folium.FeatureGroup( name = 'Applicant Deficit', control = True )


# Helath establishment groups

### Applicant Surplus
for _, row in inst_sup.iterrows():

    folium.Circle( [row[ 'nlat_ie' ], row[ 'nlong_ie' ] ] , 
                    radius= row['ratio_applicants_vacancies'] * 10000 , 
                    color = "green",
                    fill_color="green", # divvy color
                   ).add_to( inst_sup_fm )
    
# ### Applicant Deficit
# for _, row in inst_def.iterrows():

#     folium.Circle( [row[ 'nlat_ie' ], row[ 'nlong_ie' ] ] , 
#                     radius= row['ratio_applicants_vacancies'] * 10000 , 
#                     color = "red",
#                     fill_color="red", # divvy color
#                    ).add_to( inst_def_fm )


# Add groups 
m.add_child( inst_sup_fm )
# m.add_child( inst_def_fm )     
   


# legend_html_0 = '''
# <div style="
# position: fixed; 
# bottom: 10px;
# left: 10px;
# width: 200px;
# height: 120px;
# z-index:9999;
# font-size:14px;
# <p><a style="color:black;font-size:100%;margin-left:5px;">&#9658;</a>&emsp;<b>Legend</b></p>
# <p><a style="color:green;font-size:100%;margin-left:5px;">&#9679;</a>&emsp;<b>Applicant Surplus</b></p>
# <p><a style="color:red;font-size:100%;margin-left:5px;">&#9679;</a>&emsp;<b>Applicant Deficit</b></p>


# </div>
# <div style="
# position: fixed; 
# bottom: 10px;
# left: 10px;
# width: 200px;
# height: 120px; 
# z-index:9998;
# font-size:14px;
# background-color: #ffffff;

# opacity: 0.7;
# ">
# </div>
# '''

# # Add legend
# legend_html = "{% macro html(this, kwargs) %}" + legend_html_0 + "{% endmacro %}"
# legend = branca.element.MacroElement()
# legend._template = branca.element.Template(legend_html)
# m.get_root().add_child(legend)

# Add layer
folium.LayerControl().add_to(m)
m
# Get lat and long health_center
zoom_start = 8

m = folium.Map( location = ubication, zoom_start=zoom_start)

# # Add District shapefiles
# style2 = {'fillColor': '#ffffff', 'lineColor': '#00FFFFFF'}
# folium.GeoJson( data = geo_distrito[ "geometry" ], 
#                 style_function = lambda x:style2, 
#                 name = 'District Border'
#               ).add_to( m )


# Genereate groups in folioum
health_centers_group = folium.FeatureGroup( name = 'Heakth Centers', control=True)

inst_sup_fm = folium.FeatureGroup( name = 'Applicant Surplus', control = True )
inst_def_fm = folium.FeatureGroup( name = 'Applicant Deficit', control = True )


# Helath establishment groups

### Applicant Surplus
# for _, row in inst_sup.iterrows():

#     folium.Circle( [row[ 'nlat_ie' ], row[ 'nlong_ie' ] ] , 
#                     radius= row['ratio_applicants_vacancies'] * 10000 , 
#                     color = "green",
#                     fill_color="green", # divvy color
#                    ).add_to( inst_sup_fm )
    
### Applicant Deficit
for _, row in inst_def.iterrows():

    folium.Circle( [row[ 'nlat_ie' ], row[ 'nlong_ie' ] ] , 
                    radius= row['ratio_applicants_vacancies'] * 10000 , 
                    color = "red",
                    fill_color="red", # divvy color
                   ).add_to( inst_def_fm )


# Add groups 
# m.add_child( inst_sup_fm )
m.add_child( inst_def_fm )     
   


# legend_html_0 = '''
# <div style="
# position: fixed; 
# bottom: 10px;
# left: 10px;
# width: 200px;
# height: 120px;
# z-index:9999;
# font-size:14px;
# <p><a style="color:black;font-size:100%;margin-left:5px;">&#9658;</a>&emsp;<b>Legend</b></p>
# <p><a style="color:green;font-size:100%;margin-left:5px;">&#9679;</a>&emsp;<b>Applicant Surplus</b></p>
# <p><a style="color:red;font-size:100%;margin-left:5px;">&#9679;</a>&emsp;<b>Applicant Deficit</b></p>


# </div>
# <div style="
# position: fixed; 
# bottom: 10px;
# left: 10px;
# width: 200px;
# height: 120px; 
# z-index:9998;
# font-size:14px;
# background-color: #ffffff;

# opacity: 0.7;
# ">
# </div>
# '''

# # Add legend
# legend_html = "{% macro html(this, kwargs) %}" + legend_html_0 + "{% endmacro %}"
# legend = branca.element.MacroElement()
# legend._template = branca.element.Template(legend_html)
# m.get_root().add_child(legend)

# Add layer
folium.LayerControl().add_to(m)
m
# Get lat and long health_center
zoom_start = 8

m = folium.Map( location = ubication, zoom_start=zoom_start)

# Add District shapefiles
# style2 = {'fillColor': '#ffffff', 'lineColor': '#00FFFFFF'}
# folium.GeoJson( data = geo_distrito[ "geometry" ], 
#                 style_function = lambda x:style2, 
#                 name = 'District Border'
#               ).add_to( m )


# Genereate groups in folioum
health_centers_group = folium.FeatureGroup( name = 'Heakth Centers', control=True)

inst_sup_fm = folium.FeatureGroup( name = 'Applicant Surplus', control = True )
inst_def_fm = folium.FeatureGroup( name = 'Applicant Deficit', control = True )


# Helath establishment groups

### Applicant Surplus
for _, row in inst_sup.iterrows():

    folium.Circle( [row[ 'nlat_ie' ], row[ 'nlong_ie' ] ] , 
                    radius= row['ratio_applicants_vacancies'] * 10000 , 
                    color = "green",
                    fill_color="green", # divvy color
                   ).add_to( inst_sup_fm )
    
### Applicant Deficit
for _, row in inst_def.iterrows():

    folium.Circle( [row[ 'nlat_ie' ], row[ 'nlong_ie' ] ] , 
                    radius= row['ratio_applicants_vacancies'] * 10000 , 
                    color = "red",
                    fill_color="red", # divvy color
                   ).add_to( inst_def_fm )


# Add groups 
m.add_child( inst_sup_fm )
m.add_child( inst_def_fm )     
   


legend_html_0 = '''
<div style="
position: fixed; 
bottom: 10px;
left: 10px;
width: 200px;
height: 120px;
z-index:9999;
font-size:14px;
<p><a style="color:black;font-size:100%;margin-left:5px;">&#9658;</a>&emsp;<b>Legend</b></p>
<p><a style="color:green;font-size:100%;margin-left:5px;">&#9679;</a>&emsp;<b>Applicant Surplus</b></p>
<p><a style="color:red;font-size:100%;margin-left:5px;">&#9679;</a>&emsp;<b>Applicant Deficit</b></p>


</div>
<div style="
position: fixed; 
bottom: 10px;
left: 10px;
width: 200px;
height: 120px; 
z-index:9998;
font-size:14px;
background-color: #ffffff;

opacity: 0.7;
">
</div>
'''

# Add legend
legend_html = "{% macro html(this, kwargs) %}" + legend_html_0 + "{% endmacro %}"
legend = branca.element.MacroElement()
legend._template = branca.element.Template(legend_html)
m.get_root().add_child(legend)

# Add layer
folium.LayerControl().add_to(m)
m